#!/bin/sh

. "$SPEC_PATH/abstract/dbops-restart"

. "$SPEC_PATH/abstract/metrics"

e2e_test_extra_hash() {
  "$SHELL" "$PROJECT_PATH/stackgres-k8s/ci/build/build-functions.sh" path_hash \
    "$(realpath --relative-to "$PROJECT_PATH" "$SPEC_PATH/abstract/dbops-restart")"
  "$SHELL" "$PROJECT_PATH/stackgres-k8s/ci/build/build-functions.sh" path_hash \
    "$(realpath --relative-to "$PROJECT_PATH" "$SPEC_PATH/abstract/metrics")"
}

e2e_test_install() {
  NODE_LABEL_KEY="$(random_string)"
  NODE_LABEL_VALUE="$(random_string)"

  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1"

  wait_pods_running "$CLUSTER_NAMESPACE" "1"

  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=sync \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=2 \
    --set cluster.replication.groups[0].instances=1 \
    --set cluster.replication.groups[0].role=none

  deploy_curl_pod "$CLUSTER_NAMESPACE"

  wait_pods_running "$CLUSTER_NAMESPACE" "4"
  wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  switch_cluster_to_first "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
}

e2e_test() {
  run_test "Checking that synchronous replication is working" check_sync_replication_is_working

  run_test "Checking that metrics are exported" check_metrics

  run_test "Checking that replica in ha-read group is exposed in replicas service" check_replica_in_ha_read_group_not_in_replicas_service

  run_test "Checking that replica in none group is not exposed in replicas service" check_replica_in_none_group_not_in_replicas_service

  run_test "Checking that replica in none group is not elegible as primary" check_replica_in_none_group_not_elegible_as_primary

  run_test "Checking that replica in readonly group is exposed in replicas service after changing role" check_replica_in_readonly_group_in_replicas_service_after_changing_role

  run_test "Checking that old primary in ha group is not exposed in replicas service after changing role" check_old_primary_in_ha_group_not_in_replicas_service_after_changing_role

  run_test "Checking that strict synchronous replication is working" check_strict_sync_replication_is_working

  run_test "Checking that synchronous all replication is working" check_sync_all_replication_is_working

  run_test "Checking that strict synchronous all replication is working" check_strict_sync_all_replication_is_working

  run_test "Checking that cluster can restart without replicas in any ha or ha-read group" check_cluster_can_restart_without_replicas_in_any_ha_group
}

check_sync_replication_is_working() {
  check_connectivity -i 0

  if wait_until eval 'kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
    psql -q -At -c "SHOW synchronous_standby_names" | grep -q '"'"'^2 ("[^"]\+","[^"]\+")$'"'"
  then
    success "sync replication is set for primary"
  else
    fail "sync replication is not set for primary"
  fi

  local RESULT EXIT_CODE
  try_function run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE DATABASE test;"
  if "$RESULT"
  then
    success "It's possible to create a database in the primary node"
  else
    fail "It should be possible to create a database in the primary node" 
  fi

  check_service_connectivity -i 0 -h "$CLUSTER_NAME-replicas"
  try_function run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "CREATE TABLE fibonacci(num integer);" -d test > "${LOG_PATH}/test1.log"
  if "$RESULT"
  then
    fail "It's possible to create a table in the replica node"
  else
    success "It's not possible to create a table in the replica node" 
  fi

  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE TABLE fibonacci(num integer);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (1);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (2);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (3);" -d test

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
  REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "123" ]
  then
    success "inserts on the primary with sync replication where successful."
  else
    fail "inserts on the primary with sync replication where not successful."
  fi

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "$(echo "$REPLICA_RESPONSE" | tr -d '\n')" ]
  then
    success "sync replication is working"
  else
    fail "sync replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi
}

check_replica_in_ha_read_group_not_in_replicas_service() {
  local RESULT EXIT_CODE
  try_function wait_until eval 'check_pod_ip_exposed_by_service "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" "$CLUSTER_NAME-replicas"'
  if "$RESULT"
  then
    success "first replica IP was exposed by replicas sevices"
  else
    fail "first replica IP was not exposed by replicas sevices"
  fi
}

check_replica_in_none_group_not_in_replicas_service() {
  local RESULT EXIT_CODE
  try_function wait_until eval 'check_pod_ip_not_exposed_by_service "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2" "$CLUSTER_NAME-replicas"'
  if "$RESULT"
  then
    success "second replica IP was not exposed by replicas sevices"
  else
    fail "second replica IP was exposed by replicas sevices"
  fi
}

check_replica_in_none_group_not_elegible_as_primary() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=sync \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=2 \
    --set cluster.replication.groups[0].instances=1 \
    --set cluster.replication.groups[0].role=none \
    --set cluster.pods.scheduling.nodeSelector."$NODE_LABEL_KEY=$NODE_LABEL_VALUE"
  wait_until eval '[ $(kubectl get pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2" --template "{{ .metadata.labels.nofailover }}") = true ]'
  kubectl delete pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0"
  local RESULT EXIT_CODE
  try_function wait_until eval 'kubectl get pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" --template "{{ .metadata.labels.role }}" | grep -q "\(primary\|master\)"'
  if "$RESULT"
  then
    success "first replica IP was elected as primary"
  else
    fail "first replica IP was not elected as primary"
  fi
  kubectl delete pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1"
  local RESULT EXIT_CODE
  try_function wait_until -t 10 eval 'kubectl get pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2" --template "{{ .metadata.labels.role }}" | grep -q "\(primary\|master\)"'
  if "$RESULT"
  then
    fail "second replica IP was elected as primary"
  else
    success "second replica IP was not elected as primary"
  fi
}

check_replica_in_readonly_group_in_replicas_service_after_changing_role() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=sync \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=2 \
    --set cluster.replication.groups[0].instances=1 \
    --set cluster.replication.groups[0].role=readonly \
    --set cluster.pods.scheduling.nodeSelector."$NODE_LABEL_KEY=$NODE_LABEL_VALUE"
  local RESULT EXIT_CODE
  try_function wait_until eval 'check_pod_ip_exposed_by_service "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2" "$CLUSTER_NAME-replicas"'
  if "$RESULT"
  then
    success "second replica IP was exposed by replicas sevices"
  else
    fail "second replica IP was not exposed by replicas sevices"
  fi
}

check_old_primary_in_ha_group_not_in_replicas_service_after_changing_role() {
  kubectl delete pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2"
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=sync \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=2 \
    --set cluster.replication.groups[0].instances=1 \
    --set cluster.replication.groups[0].role=ha \
    --set cluster.pods.scheduling.nodeSelector."$NODE_LABEL_KEY=null"
  # This is not a typo, helm does not support removing a property so we need to repeat the change in order to make it work :facepalm:
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=sync \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=2 \
    --set cluster.replication.groups[0].instances=1 \
    --set cluster.replication.groups[0].role=ha \
    --set cluster.pods.scheduling.nodeSelector."$NODE_LABEL_KEY=null"
  wait_until delete_pod_and_wait_scheduled "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0"
  wait_until delete_pod_and_wait_scheduled "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1"
  wait_pods_running "$CLUSTER_NAMESPACE" "3"
  wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  switch_cluster_to_first "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  local RESULT EXIT_CODE
  try_function wait_until eval 'check_pod_ip_not_exposed_by_service "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" "$CLUSTER_NAME-replicas"'
  if "$RESULT"
  then
    success "old primary IP was not exposed by replicas sevices"
  else
    fail "old primary IP was exposed by replicas sevices"
  fi
  wait_until delete_pod_and_wait_scheduled "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2"
  wait_pods_running "$CLUSTER_NAMESPACE" "4"
}

check_strict_sync_replication_is_working() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=strict-sync \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=2 \
    --set cluster.replication.groups[0].instances=2 \
    --set cluster.replication.groups[0].role=readonly \
    --set cluster.pods.scheduling.nodeSelector=null
  local RESULT EXIT_CODE
  try_function wait_until -t 10 eval 'kubectl wait -n "$CLUSTER_NAMESPACE" sgcluster "$CLUSTER_NAME" --for=condition=PendingRestart --timeout 0'
  if "$RESULT"
  then
    fail "cluster is pending restart after changing the replication mode"
  else
    success "cluster is not pending restart after changing the replication mode"
  fi

  local RESULT EXIT_CODE
  try_function wait_until eval 'kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
    psql -q -At -c "SHOW synchronous_standby_names" | grep -q "^2 (\"[^\"]\+\",\"[^\"]\+\")$"'
  if "$RESULT"
  then
    success "strict sync replication is set for primary"
  else
    fail "strict sync replication is not set for primary"
  fi

  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE TABLE fibonacci_strict(num integer);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci_strict(num) VALUES (1);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci_strict(num) VALUES (2);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci_strict(num) VALUES (3);" -d test

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci_strict ORDER BY num;" -d "test")"
  REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci_strict ORDER BY num;" -d "test")"

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "123" ]
  then
    success "inserts on the primary with strict sync replication where successful."
  else
    fail "inserts on the primary with strict sync replication where not successful."
  fi

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "$(echo "$REPLICA_RESPONSE" | tr -d '\n')" ]
  then
    success "strict sync replication is working"
  else
    fail "strict sync replication is not working. The records don't match between primary and replica for the fibonacci_strict table"
  fi
}

check_sync_all_replication_is_working() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=sync-all \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=1 \
    --set cluster.replication.groups[0].instances=2 \
    --set cluster.replication.groups[0].role=readonly \
    --set cluster.pods.scheduling.nodeSelector=null
  check_connectivity -i 0

  if wait_until eval 'kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
    psql -q -At -c "SHOW synchronous_standby_names" | grep -q '"'"'^2 ("[^"]\+","[^"]\+")$'"'"
  then
    success "sync all replication is set for primary"
  else
    fail "sync all replication is not set for primary"
  fi

  local RESULT EXIT_CODE
  try_function run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE DATABASE test_all;"
  if "$RESULT"
  then
    success "It's possible to create a database in the primary node"
  else
    fail "It should be possible to create a database in the primary node" 
  fi

  check_service_connectivity -i 0 -h "$CLUSTER_NAME-replicas"
  try_function run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "CREATE TABLE fibonacci(num integer);" -d test_all > "${LOG_PATH}/test_all1.log"
  if "$RESULT"
  then
    fail "It's possible to create a table in the replica node"
  else
    success "It's not possible to create a table in the replica node" 
  fi

  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE TABLE fibonacci(num integer);" -d test_all
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (1);" -d test_all
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (2);" -d test_all
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (3);" -d test_all

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test_all")"
  REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test_all")"

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "123" ]
  then
    success "inserts on the primary with sync replication where successful."
  else
    fail "inserts on the primary with sync replication where not successful."
  fi

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "$(echo "$REPLICA_RESPONSE" | tr -d '\n')" ]
  then
    success "sync replication is working"
  else
    fail "sync replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi
}

check_strict_sync_all_replication_is_working() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set cluster.replication.mode=strict-sync-all \
    --set cluster.replication.role=ha-read \
    --set cluster.replication.syncInstances=1 \
    --set cluster.replication.groups[0].instances=2 \
    --set cluster.replication.groups[0].role=readonly \
    --set cluster.pods.scheduling.nodeSelector=null
  local RESULT EXIT_CODE
  try_function wait_until -t 10 eval 'kubectl wait -n "$CLUSTER_NAMESPACE" sgcluster "$CLUSTER_NAME" --for=condition=PendingRestart --timeout 0'
  if "$RESULT"
  then
    fail "cluster is pending restart after changing the replication mode"
  else
    success "cluster is not pending restart after changing the replication mode"
  fi

  local RESULT EXIT_CODE
  try_function wait_until eval 'kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
    psql -q -At -c "SHOW synchronous_standby_names" | grep -q "^2 (\"[^\"]\+\",\"[^\"]\+\")$"'
  if "$RESULT"
  then
    success "strict sync all replication is set for primary"
  else
    fail "strict sync all replication is not set for primary"
  fi

  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE TABLE fibonacci_strict(num integer);" -d test_all
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci_strict(num) VALUES (1);" -d test_all
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci_strict(num) VALUES (2);" -d test_all
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci_strict(num) VALUES (3);" -d test_all

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci_strict ORDER BY num;" -d "test_all")"
  REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci_strict ORDER BY num;" -d "test_all")"

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "123" ]
  then
    success "inserts on the primary with strict sync all replication where successful."
  else
    fail "inserts on the primary with strict sync all replication where not successful."
  fi

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "$(echo "$REPLICA_RESPONSE" | tr -d '\n')" ]
  then
    success "strict sync all replication is working"
  else
    fail "strict sync all replication is not working. The records don't match between primary and replica for the fibonacci_strict table"
  fi
}

check_cluster_can_restart_without_replicas_in_any_ha_group() {
  trigger_cluster_require_restart
  DBOPS_NAME="$(get_sgdbops_name restart)"

  set_restarted_pods "$CLUSTER_NAME-0 $CLUSTER_NAME-1 $CLUSTER_NAME-2"

  cat << EOF | kubectl create -f -
apiVersion: stackgres.io/v1
kind: SGDbOps
metadata:
  name: $DBOPS_NAME
  namespace: $CLUSTER_NAMESPACE
spec:
  sgCluster: $CLUSTER_NAME
  op: restart
  restart:
    method: InPlace
EOF

  check_restart_without_data

  kubectl delete sgdbops -n "$CLUSTER_NAMESPACE" "$DBOPS_NAME"
}

check_pod_ip_exposed_by_service() {
  local CLUSTER_NAMESPACE="$1"
  local POD_NAME="$2"
  local SERVICE_NAME="$3"
  local POD_IP SERVICE_IPS
  POD_IP="$(kubectl get pod -n "$CLUSTER_NAMESPACE" "$POD_NAME" --template '{{ .status.podIP }}')"
  SERVICE_IPS="$(kubectl get endpoints -n "$CLUSTER_NAMESPACE" "$SERVICE_NAME" \
    --template '{{ range .subsets }}{{ range .addresses }}{{ printf "%s\n" .ip }}{{ end }}{{ end }}')"
  if echo "$SERVICE_IPS" | grep -qxF "$POD_IP"
  then
    success "pod $POD_NAME IP was exposed by sevice $SERVICE_NAME"
  else
    fail "pod $POD_NAME IP was not exposed by sevice $SERVICE_NAME"
  fi
}

check_pod_ip_not_exposed_by_service() {
  local CLUSTER_NAMESPACE="$1"
  local POD_NAME="$2"
  local SERVICE_NAME="$3"
  local POD_IP SERVICE_IPS
  POD_IP="$(kubectl get pod -n "$CLUSTER_NAMESPACE" "$POD_NAME" --template '{{ .status.podIP }}')"
  SERVICE_IPS="$(kubectl get endpoints -n "$CLUSTER_NAMESPACE" "$SERVICE_NAME" \
    --template '{{ range .subsets }}{{ range .addresses }}{{ printf "%s\n" .ip }}{{ end }}{{ end }}')"
  if echo "$SERVICE_IPS" | grep -qxF "$POD_IP"
  then
    fail "pod $POD_NAME IP was exposed by sevice $SERVICE_NAME"
  else
    success "pod $POD_NAME IP was not exposed by sevice $SERVICE_NAME"
  fi
}

delete_pod_and_wait_scheduled() {
  local CLUSTER_NAMESPACE="$1"
  local POD_NAME="$2"
  if ! kubectl wait -n "$CLUSTER_NAMESPACE" pod "$POD_NAME" --for=condition=PodScheduled --timeout 0
  then
    kubectl delete pod -n "$CLUSTER_NAMESPACE" "$POD_NAME"
    return 1
  fi
}